{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.applications import inception_v3\n",
    "from keras import backend as K\n",
    "from keras.applications.imagenet_utils import decode_predictions\n",
    "from keras.preprocessing import image\n",
    "import numpy as np\n",
    "K.set_learning_phase(0)\n",
    "\n",
    "model = inception_v3.InceptionV3(weights='imagenet',include_top=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_1\n",
      "conv2d_1\n",
      "batch_normalization_1\n",
      "activation_1\n",
      "conv2d_2\n",
      "batch_normalization_2\n",
      "activation_2\n",
      "conv2d_3\n",
      "batch_normalization_3\n",
      "activation_3\n",
      "max_pooling2d_1\n",
      "conv2d_4\n",
      "batch_normalization_4\n",
      "activation_4\n",
      "conv2d_5\n",
      "batch_normalization_5\n",
      "activation_5\n",
      "max_pooling2d_2\n",
      "conv2d_9\n",
      "batch_normalization_9\n",
      "activation_9\n",
      "conv2d_7\n",
      "conv2d_10\n",
      "batch_normalization_7\n",
      "batch_normalization_10\n",
      "activation_7\n",
      "activation_10\n",
      "average_pooling2d_1\n",
      "conv2d_6\n",
      "conv2d_8\n",
      "conv2d_11\n",
      "conv2d_12\n",
      "batch_normalization_6\n",
      "batch_normalization_8\n",
      "batch_normalization_11\n",
      "batch_normalization_12\n",
      "activation_6\n",
      "activation_8\n",
      "activation_11\n",
      "activation_12\n",
      "mixed0\n",
      "conv2d_16\n",
      "batch_normalization_16\n",
      "activation_16\n",
      "conv2d_14\n",
      "conv2d_17\n",
      "batch_normalization_14\n",
      "batch_normalization_17\n",
      "activation_14\n",
      "activation_17\n",
      "average_pooling2d_2\n",
      "conv2d_13\n",
      "conv2d_15\n",
      "conv2d_18\n",
      "conv2d_19\n",
      "batch_normalization_13\n",
      "batch_normalization_15\n",
      "batch_normalization_18\n",
      "batch_normalization_19\n",
      "activation_13\n",
      "activation_15\n",
      "activation_18\n",
      "activation_19\n",
      "mixed1\n",
      "conv2d_23\n",
      "batch_normalization_23\n",
      "activation_23\n",
      "conv2d_21\n",
      "conv2d_24\n",
      "batch_normalization_21\n",
      "batch_normalization_24\n",
      "activation_21\n",
      "activation_24\n",
      "average_pooling2d_3\n",
      "conv2d_20\n",
      "conv2d_22\n",
      "conv2d_25\n",
      "conv2d_26\n",
      "batch_normalization_20\n",
      "batch_normalization_22\n",
      "batch_normalization_25\n",
      "batch_normalization_26\n",
      "activation_20\n",
      "activation_22\n",
      "activation_25\n",
      "activation_26\n",
      "mixed2\n",
      "conv2d_28\n",
      "batch_normalization_28\n",
      "activation_28\n",
      "conv2d_29\n",
      "batch_normalization_29\n",
      "activation_29\n",
      "conv2d_27\n",
      "conv2d_30\n",
      "batch_normalization_27\n",
      "batch_normalization_30\n",
      "activation_27\n",
      "activation_30\n",
      "max_pooling2d_3\n",
      "mixed3\n",
      "conv2d_35\n",
      "batch_normalization_35\n",
      "activation_35\n",
      "conv2d_36\n",
      "batch_normalization_36\n",
      "activation_36\n",
      "conv2d_32\n",
      "conv2d_37\n",
      "batch_normalization_32\n",
      "batch_normalization_37\n",
      "activation_32\n",
      "activation_37\n",
      "conv2d_33\n",
      "conv2d_38\n",
      "batch_normalization_33\n",
      "batch_normalization_38\n",
      "activation_33\n",
      "activation_38\n",
      "average_pooling2d_4\n",
      "conv2d_31\n",
      "conv2d_34\n",
      "conv2d_39\n",
      "conv2d_40\n",
      "batch_normalization_31\n",
      "batch_normalization_34\n",
      "batch_normalization_39\n",
      "batch_normalization_40\n",
      "activation_31\n",
      "activation_34\n",
      "activation_39\n",
      "activation_40\n",
      "mixed4\n",
      "conv2d_45\n",
      "batch_normalization_45\n",
      "activation_45\n",
      "conv2d_46\n",
      "batch_normalization_46\n",
      "activation_46\n",
      "conv2d_42\n",
      "conv2d_47\n",
      "batch_normalization_42\n",
      "batch_normalization_47\n",
      "activation_42\n",
      "activation_47\n",
      "conv2d_43\n",
      "conv2d_48\n",
      "batch_normalization_43\n",
      "batch_normalization_48\n",
      "activation_43\n",
      "activation_48\n",
      "average_pooling2d_5\n",
      "conv2d_41\n",
      "conv2d_44\n",
      "conv2d_49\n",
      "conv2d_50\n",
      "batch_normalization_41\n",
      "batch_normalization_44\n",
      "batch_normalization_49\n",
      "batch_normalization_50\n",
      "activation_41\n",
      "activation_44\n",
      "activation_49\n",
      "activation_50\n",
      "mixed5\n",
      "conv2d_55\n",
      "batch_normalization_55\n",
      "activation_55\n",
      "conv2d_56\n",
      "batch_normalization_56\n",
      "activation_56\n",
      "conv2d_52\n",
      "conv2d_57\n",
      "batch_normalization_52\n",
      "batch_normalization_57\n",
      "activation_52\n",
      "activation_57\n",
      "conv2d_53\n",
      "conv2d_58\n",
      "batch_normalization_53\n",
      "batch_normalization_58\n",
      "activation_53\n",
      "activation_58\n",
      "average_pooling2d_6\n",
      "conv2d_51\n",
      "conv2d_54\n",
      "conv2d_59\n",
      "conv2d_60\n",
      "batch_normalization_51\n",
      "batch_normalization_54\n",
      "batch_normalization_59\n",
      "batch_normalization_60\n",
      "activation_51\n",
      "activation_54\n",
      "activation_59\n",
      "activation_60\n",
      "mixed6\n",
      "conv2d_65\n",
      "batch_normalization_65\n",
      "activation_65\n",
      "conv2d_66\n",
      "batch_normalization_66\n",
      "activation_66\n",
      "conv2d_62\n",
      "conv2d_67\n",
      "batch_normalization_62\n",
      "batch_normalization_67\n",
      "activation_62\n",
      "activation_67\n",
      "conv2d_63\n",
      "conv2d_68\n",
      "batch_normalization_63\n",
      "batch_normalization_68\n",
      "activation_63\n",
      "activation_68\n",
      "average_pooling2d_7\n",
      "conv2d_61\n",
      "conv2d_64\n",
      "conv2d_69\n",
      "conv2d_70\n",
      "batch_normalization_61\n",
      "batch_normalization_64\n",
      "batch_normalization_69\n",
      "batch_normalization_70\n",
      "activation_61\n",
      "activation_64\n",
      "activation_69\n",
      "activation_70\n",
      "mixed7\n",
      "conv2d_73\n",
      "batch_normalization_73\n",
      "activation_73\n",
      "conv2d_74\n",
      "batch_normalization_74\n",
      "activation_74\n",
      "conv2d_71\n",
      "conv2d_75\n",
      "batch_normalization_71\n",
      "batch_normalization_75\n",
      "activation_71\n",
      "activation_75\n",
      "conv2d_72\n",
      "conv2d_76\n",
      "batch_normalization_72\n",
      "batch_normalization_76\n",
      "activation_72\n",
      "activation_76\n",
      "max_pooling2d_4\n",
      "mixed8\n",
      "conv2d_81\n",
      "batch_normalization_81\n",
      "activation_81\n",
      "conv2d_78\n",
      "conv2d_82\n",
      "batch_normalization_78\n",
      "batch_normalization_82\n",
      "activation_78\n",
      "activation_82\n",
      "conv2d_79\n",
      "conv2d_80\n",
      "conv2d_83\n",
      "conv2d_84\n",
      "average_pooling2d_8\n",
      "conv2d_77\n",
      "batch_normalization_79\n",
      "batch_normalization_80\n",
      "batch_normalization_83\n",
      "batch_normalization_84\n",
      "conv2d_85\n",
      "batch_normalization_77\n",
      "activation_79\n",
      "activation_80\n",
      "activation_83\n",
      "activation_84\n",
      "batch_normalization_85\n",
      "activation_77\n",
      "mixed9_0\n",
      "concatenate_1\n",
      "activation_85\n",
      "mixed9\n",
      "conv2d_90\n",
      "batch_normalization_90\n",
      "activation_90\n",
      "conv2d_87\n",
      "conv2d_91\n",
      "batch_normalization_87\n",
      "batch_normalization_91\n",
      "activation_87\n",
      "activation_91\n",
      "conv2d_88\n",
      "conv2d_89\n",
      "conv2d_92\n",
      "conv2d_93\n",
      "average_pooling2d_9\n",
      "conv2d_86\n",
      "batch_normalization_88\n",
      "batch_normalization_89\n",
      "batch_normalization_92\n",
      "batch_normalization_93\n",
      "conv2d_94\n",
      "batch_normalization_86\n",
      "activation_88\n",
      "activation_89\n",
      "activation_92\n",
      "activation_93\n",
      "batch_normalization_94\n",
      "activation_86\n",
      "mixed9_1\n",
      "concatenate_2\n",
      "activation_94\n",
      "mixed10\n",
      "avg_pool\n",
      "predictions\n"
     ]
    }
   ],
   "source": [
    "for layer in model.layers:\n",
    "    #if layer.name.startswith(\"activation_\"):\n",
    "    print (layer.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2048, 1000)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(model.layers[len(model.layers)-1].get_weights()[0]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "activation_layers = [ layer.output for layer in model.layers if layer.name.startswith(\"activation_\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['activation_1',\n",
       " 'activation_2',\n",
       " 'activation_3',\n",
       " 'activation_4',\n",
       " 'activation_5',\n",
       " 'activation_9',\n",
       " 'activation_7',\n",
       " 'activation_10',\n",
       " 'activation_6',\n",
       " 'activation_8',\n",
       " 'activation_11',\n",
       " 'activation_12',\n",
       " 'activation_16',\n",
       " 'activation_14',\n",
       " 'activation_17',\n",
       " 'activation_13',\n",
       " 'activation_15',\n",
       " 'activation_18',\n",
       " 'activation_19',\n",
       " 'activation_23',\n",
       " 'activation_21',\n",
       " 'activation_24',\n",
       " 'activation_20',\n",
       " 'activation_22',\n",
       " 'activation_25',\n",
       " 'activation_26',\n",
       " 'activation_28',\n",
       " 'activation_29',\n",
       " 'activation_27',\n",
       " 'activation_30',\n",
       " 'activation_35',\n",
       " 'activation_36',\n",
       " 'activation_32',\n",
       " 'activation_37',\n",
       " 'activation_33',\n",
       " 'activation_38',\n",
       " 'activation_31',\n",
       " 'activation_34',\n",
       " 'activation_39',\n",
       " 'activation_40',\n",
       " 'activation_45',\n",
       " 'activation_46',\n",
       " 'activation_42',\n",
       " 'activation_47',\n",
       " 'activation_43',\n",
       " 'activation_48',\n",
       " 'activation_41',\n",
       " 'activation_44',\n",
       " 'activation_49',\n",
       " 'activation_50',\n",
       " 'activation_55',\n",
       " 'activation_56',\n",
       " 'activation_52',\n",
       " 'activation_57',\n",
       " 'activation_53',\n",
       " 'activation_58',\n",
       " 'activation_51',\n",
       " 'activation_54',\n",
       " 'activation_59',\n",
       " 'activation_60',\n",
       " 'activation_65',\n",
       " 'activation_66',\n",
       " 'activation_62',\n",
       " 'activation_67',\n",
       " 'activation_63',\n",
       " 'activation_68',\n",
       " 'activation_61',\n",
       " 'activation_64',\n",
       " 'activation_69',\n",
       " 'activation_70',\n",
       " 'activation_73',\n",
       " 'activation_74',\n",
       " 'activation_71',\n",
       " 'activation_75',\n",
       " 'activation_72',\n",
       " 'activation_76',\n",
       " 'activation_81',\n",
       " 'activation_78',\n",
       " 'activation_82',\n",
       " 'activation_79',\n",
       " 'activation_80',\n",
       " 'activation_83',\n",
       " 'activation_84',\n",
       " 'activation_77',\n",
       " 'activation_85',\n",
       " 'activation_90',\n",
       " 'activation_87',\n",
       " 'activation_91',\n",
       " 'activation_88',\n",
       " 'activation_89',\n",
       " 'activation_92',\n",
       " 'activation_93',\n",
       " 'activation_86',\n",
       " 'activation_94']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer_names = [ layer.name for layer in model.layers if layer.name.startswith(\"activation_\")]\n",
    "layer_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from keras.models import Model\n",
    "activation_model = Model(inputs=model.input, outputs=activation_layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def preprocess_image(image_path):\n",
    "    img = image.load_img(image_path)\n",
    "    img = image.img_to_array(img)\n",
    "    #convert single image to a batch with 1 image\n",
    "    img = np.expand_dims(img, axis=0)    \n",
    "    img = inception_v3.preprocess_input(img)\n",
    "    return img\n",
    "\n",
    "base_image_path = 'original_images/labrador.jpg'\n",
    "# Load the image into a Numpy array\n",
    "img = preprocess_image(base_image_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "activations = activation_model.predict(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(len(activation_layers))\n",
    "activations[93].shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "#plt.matshow(activations[7][0, :, :, 0], cmap='viridis')\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"Number of Layers\", len(activations))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "layer_names[46]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "images_per_row = 16\n",
    "idx = 80\n",
    "# Now let's display our feature maps\n",
    "for layer_activation in activations[idx:idx+1]:\n",
    "    # This is the number of features in the feature map\n",
    "    n_features = layer_activation.shape[-1]\n",
    "    # The feature map has shape (1, size1, size2, n_features)\n",
    "    r = layer_activation.shape[1]\n",
    "    c = layer_activation.shape[2]\n",
    "    \n",
    "    # We will tile the activation channels in this matrix\n",
    "    n_cols = n_features // images_per_row\n",
    "    display_grid = np.zeros((r * n_cols, images_per_row * c))\n",
    "    print(display_grid.shape)\n",
    "    # We'll tile each filter into this big horizontal grid\n",
    "    for col in range(n_cols):\n",
    "        for row in range(images_per_row):\n",
    "            channel_image = layer_activation[0,\n",
    "            :, :,\n",
    "            col * images_per_row + row]\n",
    "            # Post-process the feature to make it visually palatable\n",
    "            channel_image -= channel_image.mean()\n",
    "            channel_image /= channel_image.std()\n",
    "            channel_image *= 64\n",
    "            channel_image += 128\n",
    "            channel_image = np.clip(channel_image, 0, 255).astype('uint8')\n",
    "            display_grid[col * r : (col + 1) * r,\n",
    "            row * c : (row + 1) * c] = channel_image\n",
    "    # Display the grid\n",
    "    scale = 1. / r\n",
    "    plt.figure(figsize=(scale * display_grid.shape[1],\n",
    "    scale * display_grid.shape[0]))\n",
    "    plt.title(layer_names[idx]+\" #filters=\"+str(n_features))\n",
    "    plt.grid(False)\n",
    "    plt.imshow(display_grid, aspect='auto', cmap='viridis')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "images_per_row = 8\n",
    "idx = 1\n",
    "# Now let's display our feature maps\n",
    "layer_activation=activations[idx]\n",
    "# This is the number of features in the feature map\n",
    "n_features = layer_activation.shape[-1]\n",
    "# The feature map has shape (1, size1, size2, n_features)\n",
    "r = layer_activation.shape[1]\n",
    "c = layer_activation.shape[2]\n",
    "    \n",
    "# We will tile the activation channels in this matrix\n",
    "n_cols = n_features // images_per_row\n",
    "display_grid = np.zeros((r * n_cols, images_per_row * c))\n",
    "print(display_grid.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN Visualization by Gradient Accent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "layer_name = 'activation_94'\n",
    "filter_index = 0\n",
    "layer_output = model.get_layer(layer_name).output\n",
    "loss = K.mean(layer_output[:, :, :, filter_index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grads = K.gradients(loss, model.input)[0]\n",
    "# We add 1e-5 before dividing so as to avoid accidentally dividing by 0.\n",
    "grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iterate = K.function([model.input], [loss, grads])\n",
    "# Let's test it:\n",
    "import numpy as np\n",
    "loss_value, grads_value = iterate([np.zeros((1, 150, 150, 3))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# We start from a gray image with some noise\n",
    "input_img_data = np.random.random((1, 150, 150, 3)) * 20 + 128.\n",
    "# Run gradient ascent for 40 steps\n",
    "step = 1. # this is the magnitude of each gradient update\n",
    "for i in range(40):\n",
    "    # Compute the loss value and gradient value\n",
    "    loss_value, grads_value = iterate([input_img_data])\n",
    "    # Here we adjust the input image in the direction that maximizes the loss\n",
    "    input_img_data += grads_value * step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def deprocess_image(x):\n",
    "    # normalize tensor: center on 0., ensure std is 0.1\n",
    "    x -= x.mean()\n",
    "    x /= (x.std() + 1e-5)\n",
    "    x *= 0.1\n",
    "    # clip to [0, 1]\n",
    "    x += 0.5\n",
    "    x = np.clip(x, 0, 1)\n",
    "    # convert to RGB array\n",
    "    x *= 255\n",
    "    x = np.clip(x, 0, 255).astype('uint8')\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generate_pattern(layer_name, filter_index, size=150):\n",
    "    # Build a loss function that maximizes the activation\n",
    "    # of the nth filter of the layer considered.\n",
    "    layer_output = model.get_layer(layer_name).output\n",
    "    loss = K.mean(layer_output[:, :, :, filter_index])\n",
    "    # Compute the gradient of the input picture wrt this loss\n",
    "    grads = K.gradients(loss, model.input)[0]\n",
    "    # Normalization trick: we normalize the gradient\n",
    "    grads /= (K.sqrt(K.mean(K.square(grads))) + 1e-5)\n",
    "    # This function returns the loss and grads given the input picture\n",
    "    iterate = K.function([model.input], [loss, grads])\n",
    "    # We start from a gray image with some noise\n",
    "    input_img_data = np.random.random((1, size, size, 3)) * 20 + 128.\n",
    "    # Run gradient ascent for 40 steps\n",
    "    step = 2.\n",
    "    for i in range(80):\n",
    "        loss_value, grads_value = iterate([input_img_data])\n",
    "        input_img_data += grads_value * step\n",
    "        img = input_img_data[0]\n",
    "    return deprocess_image(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(generate_pattern('activation_2', 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "idx = 0\n",
    "layer_name = layer_names[idx]\n",
    "images_per_row = 8\n",
    "size = 64\n",
    "margin = 5\n",
    "\n",
    "n_features = activation_layers[idx].shape[-1]\n",
    "n_cols = n_features // images_per_row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_features, n_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "# This a empty (black) image where we will store our results.\n",
    "results = np.zeros((images_per_row * size + 7 * margin, images_per_row * size + 7 * margin, 3))\n",
    "for i in range(n_cols): # iterate over the rows of our results grid\n",
    "    for j in range(images_per_row): # iterate over the columns of our results grid\n",
    "    # Generate the pattern for filter `i + (j * 8)` in `layer_name`\n",
    "        filter_img = generate_pattern(layer_name, (i * images_per_row) +j, size=size)\n",
    "        # Put the result in the square `(i, j)` of the results grid\n",
    "        horizontal_start = i * size + i * margin\n",
    "        horizontal_end = horizontal_start + size\n",
    "        vertical_start = j * size + j * margin\n",
    "        vertical_end = vertical_start + size\n",
    "        results[horizontal_start: horizontal_end, vertical_start: vertical_end, :] = filter_img\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Display the results grid\n",
    "plt.figure(figsize=(20, 20))\n",
    "plt.title(layer_names[idx]+\" #filters=\"+str(n_features))\n",
    "plt.imshow(results)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Dream"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "layer_name = 'activation_41'\n",
    "activation = model.get_layer(layer_name).output\n",
    "\n",
    "# We avoid border artifacts by only involving non-border pixels in the loss.\n",
    "scaling = K.prod(K.cast(K.shape(activation), 'float32'))\n",
    "loss = K.sum(K.square(activation[:, 2: -2, 2: -2, :])) / scaling\n",
    "\n",
    "# This holds our generated image\n",
    "dream = model.input\n",
    "\n",
    "# Compute the gradients of the dream with regard to the loss.\n",
    "grads = K.gradients(loss, dream)[0]\n",
    "\n",
    "# Normalize gradients.\n",
    "grads /= K.maximum(K.mean(K.abs(grads)), 1e-7)\n",
    "\n",
    "\n",
    "iterate_grad_ac_step = K.function([dream], [loss, grads])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gradient_ascent(x, iterations, step, max_loss=None):\n",
    "    for i in range(iterations):\n",
    "        loss_value, grad_values = iterate_grad_ac_step([x])\n",
    "        print('...Loss value at', i, ':', loss_value)\n",
    "        if max_loss is not None and loss_value > max_loss:\n",
    "            break        \n",
    "        x += step * grad_values\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import scipy\n",
    "\n",
    "def deprocess_image(x):\n",
    "    # Util function to convert a tensor into a valid image.\n",
    "    if K.image_data_format() == 'channels_first':\n",
    "        x = x.reshape((3, x.shape[2], x.shape[3]))\n",
    "        x = x.transpose((1, 2, 0))\n",
    "    else:\n",
    "        x = x.reshape((x.shape[1], x.shape[2], 3))\n",
    "    x /= 2.\n",
    "    x += 0.5\n",
    "    x *= 255.\n",
    "    x = np.clip(x, 0, 255).astype('uint8')\n",
    "    return x\n",
    "\n",
    "def resize_img(img, size):\n",
    "    img = np.copy(img)\n",
    "    factors = (1,\n",
    "               float(size[0]) / img.shape[1],\n",
    "               float(size[1]) / img.shape[2],\n",
    "               1)\n",
    "    return scipy.ndimage.zoom(img, factors, order=1)\n",
    "\n",
    "\n",
    "def save_img(img, fname):\n",
    "    pil_img = deprocess_image(np.copy(img))\n",
    "    scipy.misc. (fname, pil_img)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "num_octave = 4 # Number of scales at which to run gradient ascent\n",
    "octave_scale = 1.4 # Size ratio between scales\n",
    "iterations = 20 # Number of ascent steps per scale\n",
    "\n",
    "# If our loss gets larger than 10, \n",
    "# we will interrupt the gradient ascent process, to avoid ugly artifacts\n",
    "max_loss = 20.\n",
    "\n",
    "base_image_path = 'original_images/blue-sky-bright-clouds-125458.jpg'\n",
    "# Load the image into a Numpy array\n",
    "img = preprocess_image(base_image_path)\n",
    "print(img.shape)\n",
    "# We prepare a list of shape tuples\n",
    "# defining the different scales at which we will run gradient ascent\n",
    "original_shape = img.shape[1:3]\n",
    "successive_shapes = [original_shape]\n",
    "for i in range(1, num_octave):\n",
    "    shape = tuple([int(dim / (octave_scale ** i)) for dim in original_shape])\n",
    "    successive_shapes.append(shape)\n",
    "\n",
    "# Reverse list of shapes, so that they are in increasing order\n",
    "successive_shapes = successive_shapes[::-1]\n",
    "\n",
    "# Resize the Numpy array of the image to our smallest scale\n",
    "original_img = np.copy(img)\n",
    "shrunk_original_img = resize_img(img, successive_shapes[0])\n",
    "\n",
    "print(successive_shapes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "MAX_ITRN = 20\n",
    "MAX_LOSS = 20\n",
    "learning_rate = 0.01\n",
    "\n",
    "for shape in successive_shapes:\n",
    "    print('Processing image shape', shape)\n",
    "    img = resize_img(img, shape)\n",
    "    img = gradient_ascent(img,\n",
    "                          iterations=MAX_ITRN,\n",
    "                          step=learning_rate,\n",
    "                          max_loss=MAX_LOSS)\n",
    "    upscaled_shrunk_original_img = resize_img(shrunk_original_img, shape)\n",
    "    same_size_original = resize_img(original_img, shape)\n",
    "    lost_detail = same_size_original - upscaled_shrunk_original_img\n",
    "    print('adding lost details', lost_detail.shape)\n",
    "    img += lost_detail\n",
    "    shrunk_original_img = resize_img(original_img, shape)\n",
    "    save_img(img, fname='dream_at_scale_' + str(shape) + '.png')\n",
    "\n",
    "save_img(img, fname='final_dream.png')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
